import os
import re

# Function to clean up text
def clean_text(t):
    t1 = t.replace('\t', ' ').replace('\n', ' ')
    t2 = ''.join(char for char in t1 if ord(char) <= 127 and (char.isprintable() or char == ' '))
    return t2

# Function to process files in the directory
def clean_files_in_directory(directory):
    # Walk through the directory
    for root, dirs, files in os.walk(directory):
        for file in files:
            file_path = os.path.join(root, file)

            # Open the file and read the contents
            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
                content = f.read()

            cleaned_entries = []
            entries = content.strip().split('%')
            for entry in entries:
                source_quote_list = entry.strip().split("\n\t\t-- ")
                if (len(source_quote_list) == 2):
                    quote = clean_text(source_quote_list[0].strip())
                    source = clean_text(source_quote_list[1].strip())
                    if (len(quote) > 2 and len(source) > 2):
                        cleaned_entries.append(quote + '\n\t\t-- ' + source)
            cleaned_content = '\n%\n'.join(cleaned_entries) + '\n%\n'

            # Overwrite the original file with cleaned content
            with open(file_path, 'w', encoding='utf-8') as f:
                f.write(cleaned_content)

# Replace 'your_directory_path' with the path to the directory you want to clean
clean_files_in_directory('fortunes')
